

<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>ML Prediction, Planning and Simulation for Self-Driving &mdash; L5Kit 1.0.0 documentation</title>
  

  
  <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="_static/pygments.css" type="text/css" />

  
  
  
  

  
  <!--[if lt IE 9]>
    <script src="_static/js/html5shiv.min.js"></script>
  <![endif]-->
  
    
      <script type="text/javascript" id="documentation_options" data-url_root="./" src="_static/documentation_options.js"></script>
        <script src="_static/jquery.js"></script>
        <script src="_static/underscore.js"></script>
        <script src="_static/doctools.js"></script>
        <script src="_static/language_data.js"></script>
    
    <script type="text/javascript" src="_static/js/theme.js"></script>

    
    <link rel="index" title="Index" href="genindex.html" />
    <link rel="search" title="Search" href="search.html" />
    <link rel="next" title="API Reference" href="api_reference.html" />
    <link rel="prev" title="Welcome to L5Kit’s documentation!" href="index.html" /> 
</head>

<body class="wy-body-for-nav">

   
  <div class="wy-grid-for-nav">
    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search" >
          

          
            <a href="index.html" class="icon icon-home" alt="Documentation Home"> L5Kit
          

          
          </a>

          
            
            
          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        
        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <ul class="current">
<li class="toctree-l1 current"><a class="current reference internal" href="#">ML Prediction, Planning and Simulation for Self-Driving</a></li>
<li class="toctree-l1"><a class="reference internal" href="#examples">Examples</a></li>
<li class="toctree-l1"><a class="reference internal" href="#news">News</a></li>
<li class="toctree-l1"><a class="reference internal" href="#overview">Overview</a><ul>
<li class="toctree-l2"><a class="reference internal" href="#datasets">1. Datasets</a></li>
<li class="toctree-l2"><a class="reference internal" href="#l5kit">2. L5Kit</a></li>
<li class="toctree-l2"><a class="reference internal" href="#id1">3. Examples</a><ul>
<li class="toctree-l3"><a class="reference internal" href="#dataset-visualization">Dataset visualization</a></li>
<li class="toctree-l3"><a class="reference internal" href="#agent-motion-prediction">Agent motion prediction</a></li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="#installation">Installation</a><ul>
<li class="toctree-l2"><a class="reference internal" href="#clone-the-repo">1. Clone the repo</a></li>
<li class="toctree-l2"><a class="reference internal" href="#download-the-datasets">2. Download the datasets</a></li>
<li class="toctree-l2"><a class="reference internal" href="#install-l5kit">3. Install L5Kit</a></li>
<li class="toctree-l2"><a class="reference internal" href="#generate-l5kit-code-html-documentation-optional">4. Generate L5Kit code html documentation (optional)</a></li>
<li class="toctree-l2"><a class="reference internal" href="#run-example">5. Run example</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="#license">License</a></li>
<li class="toctree-l1"><a class="reference internal" href="#credits">Credits</a><ul>
<li class="toctree-l2"><a class="reference internal" href="#citation">Citation</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="#contact">Contact</a></li>
<li class="toctree-l1"><a class="reference internal" href="api_reference.html">API Reference</a></li>
<li class="toctree-l1"><a class="reference internal" href="data_format.html">Dataset Formats</a></li>
<li class="toctree-l1"><a class="reference internal" href="how_to_contribute.html">How to contribute</a></li>
</ul>

            
          
        </div>
        
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" aria-label="top navigation">
        
          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
          <a href="index.html">L5Kit</a>
        
      </nav>


      <div class="wy-nav-content">
        
        <div class="rst-content">
        
          















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="index.html" class="icon icon-home"></a> &raquo;</li>
        
      <li>ML Prediction, Planning and Simulation for Self-Driving</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
            
            <a href="_sources/README.md.txt" rel="nofollow"> View page source</a>
          
        
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <div class="section" id="ml-prediction-planning-and-simulation-for-self-driving">
<h1>ML Prediction, Planning and Simulation for Self-Driving<a class="headerlink" href="#ml-prediction-planning-and-simulation-for-self-driving" title="Permalink to this headline">¶</a></h1>
<p><img alt="ML prediction, planning and simulation for self-driving" src="_images/av.jpg" /></p>
<p>This repository and the associated datasets constitute a framework for developing learning-based solutions to prediction, planning and simulation problems in self-driving. State-of-the-art solutions to these problems still require significant amounts of hand-engineering and unlike, for example, perception systems, have not benefited much from deep learning and the vast amount of driving data available.</p>
<p>The purpose of this framework is to enable engineers and researchers to experiment with data-driven approaches to planning and simulation problems using real world driving data and contribute to state-of-the-art solutions.</p>
<p><img alt="Modern AV pipeline" src="_images/pipeline.png" /></p>
<p>This software is developed by Lyft Level 5 self-driving division and is <a class="reference internal" href="how_to_contribute.html"><span class="doc">open to external contributors</span></a>.</p>
</div>
<div class="section" id="examples">
<h1>Examples<a class="headerlink" href="#examples" title="Permalink to this headline">¶</a></h1>
<p>You can use this framework to build systems which:</p>
<ul class="simple">
<li><p>Turn prediction, planning and simulation problems into data problems and train them on real data.</p></li>
<li><p>Use neural networks to model key components of the Autonomous Vehicle (AV) stack.</p></li>
<li><p>Use historical observations to predict future movement of cars around an AV.</p></li>
<li><p>Plan behavior of an AV in order to imitate human driving.</p></li>
<li><p>Study the improvement in performance of these systems as the amount of data increases.</p></li>
</ul>
</div>
<div class="section" id="news">
<h1>News<a class="headerlink" href="#news" title="Permalink to this headline">¶</a></h1>
</div>
<div class="section" id="overview">
<h1>Overview<a class="headerlink" href="#overview" title="Permalink to this headline">¶</a></h1>
<p>The framework consists of three modules:</p>
<ol class="simple">
<li><p><strong>Datasets</strong> - data available for training ML models.</p></li>
<li><p><strong>L5Kit</strong> - the core library supporting functionality for reading the data and framing planning and simulation problems as ML problems.</p></li>
<li><p><strong>Examples</strong> - an ever-expanding collection of jupyter notebooks which demonstrate the use of L5Kit to solve various AV problems.</p></li>
</ol>
<div class="section" id="datasets">
<h2>1. Datasets<a class="headerlink" href="#datasets" title="Permalink to this headline">¶</a></h2>
<p>To use the framework you will need to download the Lyft Level 5 Prediction dataset from https://self-driving.lyft.com/level5/data/.
It consists of the following components:</p>
<ul class="simple">
<li><p>1000 hours of perception output logged by Lyft AVs operating in Palo Alto. This data is stored in 30 second chunks using the <a class="reference internal" href="data_format.html"><span class="doc">zarr format</span></a>.</p></li>
<li><p><a class="reference external" href="https://medium.com/lyftlevel5/semantic-maps-for-autonomous-vehicles-470830ee28b6">A hand-annotated, HD semantic map</a>. This data is stored using protobuf format.</p></li>
<li><p>A high-definition aerial map of the Palo Alto area. This image has 8cm per pixel resolution and is provided by <a class="reference external" href="https://www.nearmap.com/">NearMap</a>.</p></li>
</ul>
<p>To read more about the dataset and how it was generated, read the <a class="reference external" href="https://arxiv.org/abs/2006.14480">dataset whitepaper</a>.</p>
</div>
<div class="section" id="l5kit">
<h2>2. L5Kit<a class="headerlink" href="#l5kit" title="Permalink to this headline">¶</a></h2>
<p>L5Kit is a library which lets you:</p>
<ul class="simple">
<li><p>Load driving scenes from zarr files</p></li>
<li><p>Read semantic maps</p></li>
<li><p>Read aerial maps</p></li>
<li><p>Create birds-eye-view (BEV) images which represent a scene around an AV or another vehicle</p></li>
<li><p>Sample data</p></li>
<li><p>Train neural networks</p></li>
<li><p>Visualize results</p></li>
</ul>
</div>
<div class="section" id="id1">
<h2>3. Examples<a class="headerlink" href="#id1" title="Permalink to this headline">¶</a></h2>
<p>The <code class="docutils literal notranslate"><span class="pre">examples</span></code> folder contains examples in jupyter notebook format which you can use as a foundation for building your ML planning and simulation solutions. Currently we provide two examples, with more to come soon:</p>
<div class="section" id="dataset-visualization">
<h3>Dataset visualization<a class="headerlink" href="#dataset-visualization" title="Permalink to this headline">¶</a></h3>
<p>A tutorial on how to load and visualize samples from a dataset using L5Kit.</p>
</div>
<div class="section" id="agent-motion-prediction">
<h3>Agent motion prediction<a class="headerlink" href="#agent-motion-prediction" title="Permalink to this headline">¶</a></h3>
<p>An example of training a neural network to predict the future positions of cars nearby an AV. This example is a baseline solution for the Lyft 2020 Kaggle Motion Prediction Challenge.</p>
</div>
</div>
</div>
<div class="section" id="installation">
<h1>Installation<a class="headerlink" href="#installation" title="Permalink to this headline">¶</a></h1>
<div class="section" id="clone-the-repo">
<h2>1. Clone the repo<a class="headerlink" href="#clone-the-repo" title="Permalink to this headline">¶</a></h2>
<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span>git clone https://github.com/lyft/l5kit.git ./
</pre></div>
</div>
</div>
<div class="section" id="download-the-datasets">
<h2>2. Download the datasets<a class="headerlink" href="#download-the-datasets" title="Permalink to this headline">¶</a></h2>
<p>Register at https://self-driving.lyft.com/level5/data/ and download the <a class="reference external" href="https://tinyurl.com/lyft-prediction-dataset">2020 Lyft prediction dataset</a>. Store all files in a single folder.
The resulting directory structure should be:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">prediction</span><span class="o">-</span><span class="n">dataset</span><span class="o">/</span>
  <span class="o">+-</span> <span class="n">sample_scenes</span><span class="o">/</span>
  <span class="o">+-</span> <span class="n">scenes</span><span class="o">/</span>
  <span class="o">+-</span> <span class="n">aerial_map</span><span class="o">/</span>
  <span class="o">+-</span> <span class="n">semantic_map</span><span class="o">/</span>
</pre></div>
</div>
</div>
<div class="section" id="install-l5kit">
<h2>3. Install L5Kit<a class="headerlink" href="#install-l5kit" title="Permalink to this headline">¶</a></h2>
<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span><span class="nb">cd</span> l5kit
pip install -r requirements.txt
</pre></div>
</div>
</div>
<div class="section" id="generate-l5kit-code-html-documentation-optional">
<h2>4. Generate L5Kit code html documentation (optional)<a class="headerlink" href="#generate-l5kit-code-html-documentation-optional" title="Permalink to this headline">¶</a></h2>
<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span>sphinx-apidoc --module-first --separate -o API/ l5kit/l5kit l5kit/l5kit/tests*
sphinx-build . docs
</pre></div>
</div>
</div>
<div class="section" id="run-example">
<h2>5. Run example<a class="headerlink" href="#run-example" title="Permalink to this headline">¶</a></h2>
<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span>jupyter notebook examples/visualisation/visualise_data.ipynb
</pre></div>
</div>
</div>
</div>
<div class="section" id="license">
<h1>License<a class="headerlink" href="#license" title="Permalink to this headline">¶</a></h1>
<p>We use Apache 2 license for the code in this repo.</p>
<p><a class="reference external" href="LICENSE">License</a></p>
</div>
<div class="section" id="credits">
<h1>Credits<a class="headerlink" href="#credits" title="Permalink to this headline">¶</a></h1>
<p>The framework was developed at Lyft Level 5 and is maintained by the following authors and contributors:</p>
<ul class="simple">
<li><p><a class="reference external" href="https://www.linkedin.com/in/guido-zuidhof-377b6947/">Guido Zuidhof</a></p></li>
<li><p><a class="reference external" href="https://www.linkedin.com/in/luca-bergamini-61a510182/">Luca Bergamini</a></p></li>
<li><p><a class="reference external" href="https://www.linkedin.com/in/joust/">John Houston</a></p></li>
<li><p><a class="reference external" href="https://www.linkedin.com/in/yawei-ye-b76249b1/">Yawei Ye</a></p></li>
<li><p><a class="reference external" href="https://www.linkedin.com/in/suraj-m-s-7896b9126/">Suraj MS</a></p></li>
<li><p><a class="reference external" href="https://www.linkedin.com/in/oliver-scheel-98a048176/">Oliver Scheel</a></p></li>
<li><p><a class="reference external" href="https://www.linkedin.com/in/emil-praun-7597152/">Emil Praun</a></p></li>
<li><p><a class="reference external" href="https://www.linkedin.com/in/liam-kelly-83089435/">Liam Kelly</a></p></li>
<li><p><a class="reference external" href="https://www.linkedin.com/in/iglovikov/">Vladimir Iglovikov</a></p></li>
<li><p><a class="reference external" href="https://www.linkedin.com/in/chihchu/">Chih Hu</a></p></li>
<li><p><a class="reference external" href="https://www.linkedin.com/in/pondruska/">Peter Ondruska</a></p></li>
</ul>
<div class="section" id="citation">
<h2>Citation<a class="headerlink" href="#citation" title="Permalink to this headline">¶</a></h2>
<p>If you are using L5Kit or dataset in your work please cite the following <a class="reference external" href="https://arxiv.org/abs/2006.14480">whitepaper</a>:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="nd">@misc</span><span class="p">{</span><span class="n">lyft2020</span><span class="p">,</span>
<span class="n">title</span> <span class="o">=</span> <span class="p">{</span><span class="n">One</span> <span class="n">Thousand</span> <span class="ow">and</span> <span class="n">One</span> <span class="n">Hours</span><span class="p">:</span> <span class="n">Self</span><span class="o">-</span><span class="n">driving</span> <span class="n">Motion</span> <span class="n">Prediction</span> <span class="n">Dataset</span><span class="p">},</span>
<span class="n">author</span> <span class="o">=</span> <span class="p">{</span><span class="n">Houston</span><span class="p">,</span> <span class="n">J</span><span class="o">.</span> <span class="ow">and</span> <span class="n">Zuidhof</span><span class="p">,</span> <span class="n">G</span><span class="o">.</span> <span class="ow">and</span> <span class="n">Bergamini</span><span class="p">,</span> <span class="n">L</span><span class="o">.</span> <span class="ow">and</span> <span class="n">Ye</span><span class="p">,</span> <span class="n">Y</span><span class="o">.</span> <span class="ow">and</span> <span class="n">Jain</span><span class="p">,</span> <span class="n">A</span><span class="o">.</span> <span class="ow">and</span> <span class="n">Omari</span><span class="p">,</span> <span class="n">S</span><span class="o">.</span> <span class="ow">and</span> <span class="n">Iglovikov</span><span class="p">,</span> <span class="n">V</span><span class="o">.</span> <span class="ow">and</span> <span class="n">Ondruska</span><span class="p">,</span> <span class="n">P</span><span class="o">.</span><span class="p">},</span>
<span class="n">year</span> <span class="o">=</span> <span class="p">{</span><span class="mi">2020</span><span class="p">},</span>
<span class="n">howpublished</span> <span class="o">=</span> <span class="p">{</span>\<span class="n">url</span><span class="p">{</span><span class="n">https</span><span class="p">:</span><span class="o">//</span><span class="bp">self</span><span class="o">-</span><span class="n">driving</span><span class="o">.</span><span class="n">lyft</span><span class="o">.</span><span class="n">com</span><span class="o">/</span><span class="p">}}</span>
</pre></div>
</div>
<p><img alt="Lyft Level 5" src="_images/lyft.jpg" /></p>
</div>
</div>
<div class="section" id="contact">
<h1>Contact<a class="headerlink" href="#contact" title="Permalink to this headline">¶</a></h1>
<p>If you find problem or have questions about L5Kit please feel free to create <a class="reference external" href="https://github.com/lyft/l5kit/issues">github issue</a> or reach out to l5kit&#64;lyft.com!</p>
</div>


           </div>
           
          </div>
          <footer>
  
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="api_reference.html" class="btn btn-neutral float-right" title="API Reference" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a>
      
      
        <a href="index.html" class="btn btn-neutral float-left" title="Welcome to L5Kit’s documentation!" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a>
      
    </div>
  

  <hr/>

  <div role="contentinfo">
    <p>
        
        &copy; Copyright 2020, Lyft Level 5

    </p>
  </div>
    
    
    
    Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a
    
    <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a>
    
    provided by <a href="https://readthedocs.org">Read the Docs</a>. 

</footer>

        </div>
      </div>

    </section>

  </div>
  

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>

  
  
    
   

</body>
</html>